#!/usr/bin/env python

################################################################################
#   Copyright 2010 Jason Hoover
#
#
#   This file is part of AutoCache.
#
#   AutoCache is free software: you can redistribute it and/or modify
#   it under the terms of the GNU Lesser General Public License as published by
#   the Free Software Foundation, either version 3 of the License, or
#   (at your option) any later version.
#
#   AutoCache is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#   GNU General Public License for more details.

#   You should have received a copy of the GNU Lesser General Public License
#   along with AutoCache.  If not, see <http://www.gnu.org/licenses/>.
################################################################################

################################################################################
# ac-common - Contains functions every program will need, including report 
# generators or the like.
# 
# Function Name   Description                        Usage                   Returns
#
# ac_configurure  Configuration parser               ac_configure(conffile)  DICT of Settings 
# ac_stateinit    Creates state directories/files.   ac_stateinit()          Nothing
# ac_df           Generates disk free percentage.    ac_df(path)             INT Percentage of disk free
# ac_du           Lambda Runs du for specified path  ac_du(path)             INT Disk usage of path in MB
# ac_urltohash    Lambda converts a URLs to hash     ac_url(str url)         STR MD5 Hash of second part.
# ac_urltofle     Converts URLs into filenames       ac_urltofile(url)       A filename, derived from the URL
# ac_mknice       Alias to os.nice with setting.     ac_mknice()             See os.nice.
# ac_log          Wrapper of python logger           ac_log(str message)     Nothing
################################################################################

import os
import subprocess
import urlparse
import hashlib
import logging
import sys

################################################################################
# ac_configure - Loads the configuration file into memory, parses it, and reads 
# those settings into ac_settings, which it then returns as it's result. All 
# settings should be specified, loaded, and parsed here. Everything else should
# be considered temporary. Should typically only be executed once per 
# exectutable at runtime. All of the other applications should load this.
################################################################################

def ac_configure ( config_location, cmdopts="" ):
    
    # Set up sane defaults.
    # WARNING This ac_settings is local. This function returns ac_settings 
    # which must be set globally!
    #
    # Definitions of each setting in brief:
    ###########################################
    # log_file        - Destination path to a log file.
    # log_level       - Logging level, specified in python's logger class.
    # log_format      - Format of the log entries.
    # du_exec         - Path to du.
    # dl_exec         - Path to wget.
    # dl_bwlimit      - Limit of downloads in kb.
    # dl_slots        - Number of simultanious downloads.
    # dl_timeout      - Socket/wget timeout in seconds.
    # nice            - Number for wget's niceness (not implemented).
    # cache_dir       - Path to the cache directory.
    # cache_web       - http server path.
    # cache_pctlimit  - Max percentage of disk usage for cache.
    # cache_mblimit   - Max size of cache in MB.
    # cache_fmode     - Permissions in octal for cached files.
    # cache_dmode     - Permissions in octal for cache directories.
    # trust_long      - Seconds to trust fairly reliable objects.
    # trust_short     - Seconds to trust semi reliable objects.
    # state_dir       - Path to state storage.
    # targets         - Path to targets file.
    # concurrency     - Whether or not to perform concurrency to squid.
    # logger          - A logger object.
    
    ac_settings = {
        "log_file"       :False,
        "log_level"      :0,
        "log_format"     :"%(asctime)s [%(process)d:%(threadName)s] %(levelname)s %(funcName)s: %(message)s",
        "du_exec"        :"/usr/bin/du",    
        "dl_exec"        :"/usr/bin/wget",
        "dl_bwlimit"     :0,
        "dl_slots"       :1,
        "dl_timeout"     :90,
        "nice"           :0,
        "cache_dir"      :"",
        "cache_web"      :"",
        "cache_pctlimit" :90,
        "cache_mblimit"  :0,
        "cache_fmode"    :0744,
        "cache_dmode"    :0755,
        "trust_long"     :1000,
        "trust_short"    :60,
        "state_dir"      :"",
        "targets"        :"",
        "concurrency"    :False,
        "logger"         :False,
        }


    # Define words that imply true or false so they can be used in the config
    # if necessary.
    true_false = {
        "true"  : True,
        "1"     : True,
        "yes"   : True,
        "0"     : False,
        "no"    : False,
        "false" : False,
        }

    # Define the possible levels here.
    log_levels = {
        'debug'    : logging.DEBUG,
        'info'     : logging.INFO,
        'warning'  : logging.WARNING,
        'error'    : logging.ERROR,
        'critical' : logging.CRITICAL,
        }

    # Open and read the configuration file object. Die if that fails.
    try:
        config_file = open( config_location )

        # Read the file into a dict. Read line by line, strip newlines, make 
        # and split at ='s.

        config_contents = {}
        lastline = config_file.readline().rstrip().split('=')    
    
        while lastline[0] != "":            
            config_contents[lastline[0]] = lastline[1]
            lastline = config_file.readline().rstrip().split('=')

    except OSError:
        raise OSError ("FAILURE: Configuration not specified or unreadable.")
        

    # Create the logger object regardless of if it's configured. We'll be 
    # using it no matter if it's configured or not!
    ac_settings["logger"] = logging.getLogger("ac_log")

    if config_contents.has_key("log_file") and  \
       config_contents["log_file"] > "":

        log_handler=logging.FileHandler(config_contents["log_file"])

        # Set up the formatting. Has to be done before the handler is added to
        # the logger.
        if config_contents.has_key("log_format") and \
           config_contents["log_format"] > "":
                log_handler.setFormatter(logging.Formatter(config_contents["log_format"]))
        else:
                log_handler.setFormatter(logging.Formatter(ac_settings["log_format"]))


        # Add a log file handler.
        ac_settings["logger"].addHandler(log_handler)


    #If there's no log level set, set it to -everything-.
    if not config_contents.has_key("log_level") or \
       config_contents["log_level"] == "":
            ac_settings["logger"].setLevel(logger.NOTSET)

    # First test if the log level in the config is actually valid.
    elif config_contents.has_key("log_level") and \
       config_contents["log_level"] > "" and      \
       log_levels.get(config_contents["log_level"].lower()) == None:
            raise ValueError("Invalid log_level set! See README for valid levels.")

    # If so, set it.
    elif config_contents.has_key("log_level") and config_contents["log_level"] > "":

            # And then set the correct level. Note that we use NOTSET in the
            # event that there is no log level set, which by default is
            # (i think) everything but debug.
            ac_settings["logger"].setLevel(
                log_levels.get(config_contents["log_level"].lower())
                )

    # Set wget exec variable if set. 

    if config_contents.has_key("dl_exec") and  \
       os.access(config_contents["dl_exec"], os.X_OK):
            ac_settings["dl_exec"] = config_contents["dl_exec"]

    else:
        try:
            subprocess.call("wget", stdout=subprocess.PIPE)
            ac_settings["dl_exec"] = "wget"

        except OSError:
            raise OSError ("FAILURE: Cannot find wget in PATH. Buhhhh?!")


    # Same for du. (Maybe I should make a function for this.. >___>)

    if config_contents.has_key("du_exec") and  \
       os.access(config_contents["du_exec"], os.X_OK):
            ac_settings["du_exec"]=config_contents["du_exec"]
    else:
        try:
            subprocess.call(["du","/dev/null"], stdout=subprocess.PIPE)
            ac_settings["du_exec"]="du"

        except OSError:
            raise OSError("FAILURE: Cannot find du in PATH. Buhhhh?!")


    # Set the dl_bwlimit value in kilobytes.
    # If not set, leave the default value and do no further processing.
    # If higher than 0, less than 999 gigabytes per second, raise one ourselves.
    if not config_contents.has_key("dl_bwlimit"):
        pass

    elif config_contents["dl_bwlimit"].isdigit() and \
         int(config_contents["dl_bwlimit"]) > 0  and \
         int(config_contents["dl_bwlimit"]) < 999999999:
            ac_settings["dl_bwlimit"]=int(config_contents["dl_bwlimit"])

    else:
        raise ValueError("FAILURE: dl_bwlimit set, but either not a number, negative, zero, or -way- too high! Setting was: " + 
                         config_contents["dl_bwlimit"])


    # Max disk usage percent..
    if not config_contents.has_key("cache_pctlimit"):
        pass
    elif config_contents["cache_pctlimit"].isdigit() and \
         int(config_contents["cache_pctlimit"]) > 0 and  \
         int(config_contents["cache_pctlimit"]) < 101:
            ac_settings["cache_pctlimit"]=int(config_contents["cache_pctlimit"])
    else:
        raise ValueError("FAILURE: cache_pctlimit set, but either not a number, negative, zero, or more than 100. Setting was: " + 
                         config_contents["cache_pctlimit"])


    # Download slots.
    if not config_contents.has_key("dl_slots"):
        pass

    elif config_contents["dl_slots"].isdigit() and \
         int(config_contents["dl_slots"]) > 0 and  \
         int(config_contents["dl_slots"]) < 1000:
            ac_settings["dl_slots"]=int(config_contents["dl_slots"])

    else:
        raise ValueError("FAILURE: dl_slots set, but either not a number, negative, zero, or more than 100. Setting was: " + 
                         config_contents["dl_slots"])



    # Max disk usage in MB
    if config_contents.has_key("cache_mblimit") and   \
       config_contents["cache_mblimit"].isdigit() and \
       int(config_contents["cache_mblimit"]) > 0:
           ac_settings["cache_mblimit"]=int(config_contents["cache_mblimit"]) * 1024 * 1024

    else:
        raise ValueError("FAILURE: cache_mblimit either not set, not a number, negative, or zero, Setting was: " + 
                         config_contents["cache_pctlimit"])
        

    # Set up cache directory.
    if not config_contents.has_key("cache_dir"):
        raise KeyError("FAILURE: cache_dir unset! WTF am I supposed to do without one?!")

    # Remember to use -or- since these are bit comparisons.
    elif os.path.isdir(config_contents["cache_dir"]) and \
         os.access(config_contents["cache_dir"],os.R_OK | os.W_OK):

            # Add a trailing slash now so I don't have to enter it in the code
            # 15 fucking times.
            ac_settings["cache_dir"]=config_contents["cache_dir"] + "/"

    else:
        raise OSError("FAILURE: cache_dir either unreadable, unwritable, unlistable, or not actually a directory. Setting was: " +
                      config_contents["cache_dir"])


    # Set the path to the state_dir, ac_stateinit() will need this later.
    if not config_contents.has_key("state_dir"):
        raise KeyError("FAILURE: state_dir unset! Needed for most normal operations. Cannot continue.")

    # Again, remember to use or here..
    elif os.access(config_contents["state_dir"],os.R_OK | os.W_OK) and  \
         os.path.isdir(config_contents["state_dir"]):

            # Add a trailing slash now so I don't have to enter it in the code
            # 15 fucking times.
            ac_settings["state_dir"]=config_contents["state_dir"]+"/"

    else:
        raise OSError("FAILURE: state_dir either unreadable, unwritable, unlistable, or not actually a directory. Setting was: " + 
                      config_contents["state_dir"])


    # Validate the existance of the targets regex list, but don't bother 
    # parsing it.
    if not config_contents.has_key("targets"):
        raise KeyError("FAILURE: No targets file specified.")

    elif os.path.isfile(config_contents["targets"]) and  \
         os.access(config_contents["targets"], os.R_OK):
            ac_settings["targets"]=config_contents["targets"]

    else:
        raise OSError("FAILURE: Could not open targets file: " + 
                        config_contents["targets"])
        

    # Set up URL base
    try:
        if urlparse.urlparse(config_contents["cache_web"]).scheme != "http":
            raise ValueError("FAILURE: Only http destination URLs presently supported. Setting was: " +
                             config_contents["cache_web"])
            
        elif urlparse.urlparse(config_contents["cache_web"]).netloc == "":
            raise ValueError("FAILURE: Either target URL badly written, or you need at least a hostname...")

        else:
            ac_settings["cache_web"]=config_contents["cache_web"]

    except KeyError:
        raise KeyError("FAILURE: Destination URL unset.")

    
    # Lastly, validate nice, leave default if unset. From nice(1), should be
    # between -20 and 19.
    if not config_contents.has_key("nice"):
        pass    
 
    elif config_contents["nice"].isdigit() and \
        int(config_contents["nice"]) > -21 and \
        int(config_contents["nice"]) < 20:
                ac_settings["nice"] = int(config_contents["nice"])

    else:
        raise ValueError("FAILURE: Nice must be an integer between -20 and 19. Setting was: " + 
                         config_contents["nice"])



    # Configure concurrency - If set.
    if config_contents.has_key("concurrency"):
            ac_settings["concurrency"] = true_false.get(config_contents["concurrency"].lower(), False)


    # Set up cache file mode settings.
    if not config_contents.has_key("cache_fmode"):
        pass

    elif config_contents["cache_fmode"].isdigit() and   \
         int(config_contents["cache_fmode"]) < 10000:

        try:

            # Turn the STR generated by the config reader into a base 8 INT (a
            # decimalized octal) so that it can be given to os.chmod.
            ac_settings["cache_fmode"] = int(config_contents["cache_fmode"], 8)

        except ValueError:
            raise ValueError("FAILURE: cache_fmode was set, but was not a proper octal permissions value!")

    else:
        raise ValueError("FAILURE: cache_fmode was set, but the value looks really weird, setting was: " + 
                         config_contents["cache_fmode"])


    # Set up cache directory mode settings.
    if not config_contents.has_key("cache_dmode"):
        pass

    elif config_contents["cache_dmode"].isdigit() and \
         int(config_contents["cache_dmode"]) < 10000:

        try:    

            # Turn the STR generated by the config reader into a base 8 INT (a
            # decimalized octal) so that it can be given to os.chmod.
            ac_settings["cache_dmode"] = int(config_contents["cache_dmode"], 8)

        except ValueError:
             raise ValueError("FAILURE: cache_dmode was set, but was not a proper octal permissions value!")

    else:
        raise ValueError("FAILURE: cache_dmode was set, but the value looks really weird, setting was: " + 
                         config_contents["cache_dmode"])


    # Set up the default timeout value for network I/O
    if not config_contents.has_key("dl_timeout"):
        pass

    elif config_contents["dl_timeout"].isdigit() and \
         int(config_contents["dl_timeout"]) < 1000:
            ac_settings["dl_timeout"] = int(config_contents["dl_timeout"])

    else:
        raise ValueError("FAILURE: dl_timeout was set, but the value is either weird or greater than 1000, setting was: " + 
                         config_contents["dl_timeout"])


    # All done.
    return ac_settings

################################################################################
# ac_stateinit - configures the state DIR. Extracted from the original 
# configuration utility because there's sometimes a need to call this from other
# functions if the statedir is wiped or fails.
################################################################################

def ac_stateinit():

    # Let's build a list of files and directories that must exist.
    files = ( 
        "queue", 
        "lastcleanup",
        "cachesize", 
        )

    # If we have to create sub directories, make sure to write them IN ORDER.
    dirs = (
        "download", 
        "download/active",
        "download/locks",
        )

    for each in dirs:

        each=ac_settings["state_dir"]+each

        try:
            if not os.access(each, os.R_OK | os.W_OK) or not os.path.isdir(each):
                os.mkdir(each)
                ac_log.info("Needed to create state file: " + each)

        except OSError:
            # If we can't make the download DIR (something that can't ever 
            # happen, because at this point, the state_dir should exist and be
            # writable, who knows, maybe it got replaced with a file or 
            # something?
            ac_log.critical("FAILURE: Could not set up state directory: " + each)
            raise OSError("FAILURE: Could not set up state directory: " + each)


    for each in files:

        each=ac_settings["state_dir"]+each

        try:
            if not os.access(each, os.R_OK | os.W_OK) or not  \
                   os.path.isfile(each):
                    file(each,'a').close()
                    ac_log.info("Needed to create state file: " + each)

        except IOError:
            # Again, should never happen.
            ac_log.critical("FAILURE: Could not read/write/create the state file: "
                            + each)
            raise IOError("FAILURE: Could not read/write/create the state file: "
                            + each)


    return


################################################################################
# ac_diskfree - lambda function, returning free disk space of path in percent.
################################################################################

ac_df = lambda path: int(
                        float(os.statvfs(path).f_bfree) / 
                        float(os.statvfs(path).f_blocks) *
                        100 )


################################################################################
# ac_du - A terrible, complex lambda function, returning disk usage in bytes of 
# a given path. Takes a long time to execute because du is VERY I/O heavy, so it 
# should be used SPARINGLY.
#
# A quick rundown of how this works:
# -Open a subprocess for du, summary (-s) in bytes (-B 1), path.
# -Put stdout and stderr into the communicate object.
# -Start the program with ac_mknice
# -Select the stdout, which is 0
# -Split the stdout by \t.
# -Return the first chunk of the split output.
#
################################################################################

ac_du = lambda path: int( 
    subprocess.Popen(
        (
            ac_settings["du_exec"],
            "-s",
            "-B 1",
            path,
        ),
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        preexec_fn=ac_mknice,
        ).communicate()[0].split("\t")[0]
    )


################################################################################
# ac_urltohash - Another simple lambda function that turns a URL into an MD5 
# hash. This is what will be stored in the cache_dir. Standalone so that the 
# generator can be easily modified.
################################################################################

ac_urltohash = lambda url: hashlib.md5(urlparse.urlparse(url).path).hexdigest()


################################################################################
# ac_urltofile - A short, simple, stupid lambda which returns a filename for a 
# given url. Similar to urltohash, this is so that we can change the way 
# filenames are generated throughout the whole program easily.
################################################################################

ac_urltofile = lambda url: str(url).split('/')[-1]


################################################################################
# ac_mknice - Make a subprocess nice. Called as a preexec_fn for wget and du.
################################################################################

ac_mknice = lambda: os.nice(ac_settings["nice"])

################################################################################
# The functions below are executed upon import.
################################################################################

try:
    # Import the config specified on the command line.
    ac_settings = ac_configure(str(sys.argv[1]))
    ac_log = ac_settings["logger"]

except KeyError:
    raise ("No config file specified!")

ac_stateinit()
